# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

import os
import requests
# 导入访问MySQL的模块
import mysql.connector

# 保存到本地文件
class GifgetPipeline(object):
    basepath = 'D://spiderFile/gifget/'
    
    def process_item(self, item, spider):
        # 按照标题分文件夹存储
        package = item['title']
        filePath = self.basepath+package
        # 去除首位空格
        filePath=filePath.strip()
        # 去除尾部 \ 符号
        filePath=filePath.rstrip("\\")
        # 创建文件夹
        isExists=os.path.exists(filePath)
        # 如果不存在则创建目录
        if not isExists:  
            os.makedirs(filePath)
        pics = item['pics']
        # 写入图片文件
        for pic in pics:  
            r = requests.get(pic,timeout=5)
            imgaePath = filePath+pic[pic.rfind("/"):]
            with open(imgaePath,'wb') as f:
                f.write(r.content)
        return item

# 保存到数据库
class GifgetMysqlPipeline(object):
    # 数据库配置 如果要使用需改成自己数据库配置
    conf = {
        'user':'****',
        'password':'****',
        'host':'****',
        'database':'****'
    }

    # 定义构造器，初始化要写入的文件
    def __init__(self):
        self.conn = mysql.connector.connect(user=self.conf['user'], password=self.conf['password'],
            host=self.conf['host'], port='3306',
            database=self.conf['database'], use_unicode=True)
        self.cur = self.conn.cursor()
    # 重写close_spider回调方法，用于关闭数据库资源
    def close_spider(self, spider):
        print('----------关闭数据库资源-----------')
        # 关闭游标
        self.cur.close()
        # 关闭连接
        self.conn.close()
    def process_item(self, item, spider):
        pics = item['pics']
        for pic in pics:  
            print(pic)
            print(item['title'])
            print(item['publish_time'])
            self.cur.execute("INSERT INTO gifget (title,url,publish_date) VALUES(%s, %s, %s)", (item['title'],pic, item['publish_time'][0]))
            self.conn.commit()
